
<!DOCTYPE HTML>
<html lang="" >
    <head>
        <meta charset="UTF-8">
        <meta content="text/html; charset=utf-8" http-equiv="Content-Type">
        <title>Introduction · 基于spark的快速离线分析框架使用文档</title>
        <meta http-equiv="X-UA-Compatible" content="IE=edge" />
        <meta name="description" content="">
        <meta name="generator" content="GitBook 3.2.3">
        <meta name="author" content="shengshi_feiyang@yeah.net">
        
        
    
    <link rel="stylesheet" href="gitbook/style.css">

    
            
                
                <link rel="stylesheet" href="gitbook/gitbook-plugin-back-to-top-button/plugin.css">
                
            
                
                <link rel="stylesheet" href="gitbook/gitbook-plugin-cuav-chapters/cuav-chapters.css">
                
            
                
                <link rel="stylesheet" href="gitbook/gitbook-plugin-splitter/splitter.css">
                
            
                
                <link rel="stylesheet" href="gitbook/gitbook-plugin-multipart/multipart.css">
                
            
                
                <link rel="stylesheet" href="gitbook/gitbook-plugin-highlight/website.css">
                
            
                
                <link rel="stylesheet" href="gitbook/gitbook-plugin-search/search.css">
                
            
                
                <link rel="stylesheet" href="gitbook/gitbook-plugin-theme-fexa/fexa.css">
                
            
        

    

    
        
    
        
    
        
    
        
    
        
    
        
    

        
    
    
    <meta name="HandheldFriendly" content="true"/>
    <meta name="viewport" content="width=device-width, initial-scale=1, user-scalable=no">
    <meta name="apple-mobile-web-app-capable" content="yes">
    <meta name="apple-mobile-web-app-status-bar-style" content="black">
    <link rel="apple-touch-icon-precomposed" sizes="152x152" href="gitbook/images/apple-touch-icon-precomposed-152.png">
    <link rel="shortcut icon" href="gitbook/images/favicon.ico" type="image/x-icon">

    
    <link rel="next" href="develop-guide/" />
    
    

    </head>
    <body>
        
<div class="book">
	<div class="header-inner">
		<!-- LOGO -->
		<div class="logo"></div>
		<span class="title"></span>

		<!-- Search -->
		
<div id="book-search-input" role="search">
    <input type="text" placeholder="Type to search" />
</div>


		<!-- Nav -->
		<ul class="header-nav">
			<li>
				<a href="https://gitee.com/shengshifeiyang/easy-spark" target="_blank">代码仓库</a>
			</li>
		</ul>
	</div>

	<div class="book-summary">
		<div class="book-summary-title">文档目录</div>
		
		
		<nav role="navigation">
			


<ul class="summary">
    
    

    

    
        
        
    
        <li class="chapter active" data-level="1.1" data-path="./">
            
                <a href="./">
            
                    
                    Introduction
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.2" data-path="develop-guide/">
            
                <a href="develop-guide/">
            
                    
                    开发指导
            
                </a>
            

            
            <ul class="articles">
                
    
        <li class="chapter " data-level="1.2.1" data-path="develop-guide/geo-json-app.html">
            
                <a href="develop-guide/geo-json-app.html">
            
                    
                    文件读取示例hello-word
            
                </a>
            

            
        </li>
    

            </ul>
            
        </li>
    
        <li class="chapter " data-level="1.3" data-path="spark-base/">
            
                <a href="spark-base/">
            
                    
                    spark基础示例
            
                </a>
            

            
            <ul class="articles">
                
    
        <li class="chapter " data-level="1.3.1" data-path="spark-base/java-rdd.html">
            
                <a href="spark-base/java-rdd.html">
            
                    
                    JavaRDD操作示例
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.3.2" data-path="spark-base/dataset.html">
            
                <a href="spark-base/dataset.html">
            
                    
                    Dataset操作示例
            
                </a>
            

            
        </li>
    

            </ul>
            
        </li>
    
        <li class="chapter " data-level="1.4" data-path="spark-code/">
            
                <a href="spark-code/">
            
                    
                    spark常用代码片段
            
                </a>
            

            
            <ul class="articles">
                
    
        <li class="chapter " data-level="1.4.1" data-path="spark-code/map.html">
            
                <a href="spark-code/map.html">
            
                    
                    map片段
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.4.2" data-path="spark-code/flatMap.html">
            
                <a href="spark-code/flatMap.html">
            
                    
                    flatMap片段
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.4.3" data-path="spark-code/wrapped-array.html">
            
                <a href="spark-code/wrapped-array.html">
            
                    
                    WrappedArray解析片段
            
                </a>
            

            
        </li>
    

            </ul>
            
        </li>
    

    

    <li class="divider"></li>

    <li>
        <a href="https://www.gitbook.com" target="blank" class="gitbook-link">
            Published with GitBook
        </a>
    </li>
</ul>


		</nav>
		
		
	</div>

	<!-- Content nav -->
	<div class="book-anchor">
		<div class="book-anchor-title">在这篇文章中:</div>
		<div class="book-anchor-body">

		</div>
	</div>

	<div class="book-body">
		
		<div class="body-inner">
			
			

<div class="book-header" role="navigation">
    

    <!-- Title -->
    <h1>
        <i class="fa fa-circle-o-notch fa-spin"></i>
        <a href="." >Introduction</a>
    </h1>
</div>




			<div class="page-wrapper" tabindex="-1" role="main">
				<div class="page-inner">
					
<div id="book-search-results">
    <div class="search-noresults">
    
					<section class="normal markdown-section">
						
						<h1 id="&#x57FA;&#x4E8E;spark&#x7684;&#x5FEB;&#x901F;&#x79BB;&#x7EBF;&#x5206;&#x6790;&#x6846;&#x67B6;&#x4F7F;&#x7528;&#x6587;&#x6863;">&#x57FA;&#x4E8E;spark&#x7684;&#x5FEB;&#x901F;&#x79BB;&#x7EBF;&#x5206;&#x6790;&#x6846;&#x67B6;&#x4F7F;&#x7528;&#x6587;&#x6863;</h1>
<p>&#x5728;spark&#x57FA;&#x7840;&#x4E0A;,&#x505A;&#x7B80;&#x5355;&#x4E8C;&#x6B21;&#x5C01;&#x88C5;&#xFF0C;&#x7B80;&#x5316;&#x521D;&#x7EA7;&#x7A0B;&#x5E8F;&#x5458;&#x5FEB;&#x901F;&#x5165;&#x624B;spark&#x5206;&#x6790;&#x7A0B;&#x5E8F;</p>
<h2 id="&#x67B6;&#x6784;">&#x67B6;&#x6784;</h2>
<p><img src="assets/easy-spark.png" alt="&#x67B6;&#x6784;&#x56FE;"></p>
<h2 id="&#x67B6;&#x6784;&#x8BF4;&#x660E;">&#x67B6;&#x6784;&#x8BF4;&#x660E;</h2>
<ol>
<li>EngineCore&#x81EA;&#x5B9A;&#x4E49;&#x5C01;&#x88C5;&#x6838;&#x5FC3;&#x5F15;&#x64CE;</li>
<li>SparkBase&#x5F00;&#x53D1;&#x8005;&#x7EE7;&#x627F;&#x7C7B;: &#x6CE8;&#x5165;SQLContext,JavaSparkContext,SparkSession,ConfKit,DbFactory</li>
<li>SQLContext,JavaSparkContext,SparkSession:spark&#x539F;&#x751F;&#x64CD;&#x4F5C;&#x7C7B;</li>
<li>ConfKit:&#x81EA;&#x5B9A;&#x4E49;&#x914D;&#x7F6E;&#x6587;&#x4EF6;&#x64CD;&#x4F5C;&#x7C7B;config.properties</li>
<li>DbFactory:&#x6570;&#x636E;&#x4F1A;&#x8BDD;&#x5DE5;&#x5382;&#x914D;&#x7F6E;&#x5404;&#x7C7B;&#x6570;&#x636E;&#x6E90;</li>
</ol>
<h2 id="&#x672C;&#x5730;&#x5F00;&#x53D1;&#x6A21;&#x5F0F;">&#x672C;&#x5730;&#x5F00;&#x53D1;&#x6A21;&#x5F0F;</h2>
<h3 id="app&#x5165;&#x53E3;&#x7A0B;&#x5E8F;">APP&#x5165;&#x53E3;&#x7A0B;&#x5E8F;</h3>
<p>&#x901A;&#x8FC7;&#x6CE8;&#x89E3;&#x7684;@Engine(name = &quot;engineName&quot;)&#x65B9;&#x5F0F;&#x58F0;&#x660E;spark&#x7EDF;&#x8BA1;&#x4EFB;&#x52A1;&#x5E76;&#x7EE7;&#x627F;SparkBase&#x7C7B;,&#x5B9E;&#x73B0;doAi&#x65B9;&#x6CD5;&#x3002;</p>
<pre><code>@Engine(name = &quot;GeoJsonApp&quot;)
public class GeoJsonApp extends SparkBase {

    @Override
    public void doAi(JSONObject params) {
        Dataset&lt;Row&gt; jsonDataset = sparkSession.read().json(configKit.getEsparkFilePath());
        jsonDataset.printSchema();
        jsonDataset.show();
    }

}
</code></pre><p>&#x542F;&#x52A8;&#x4EFB;&#x52A1;</p>
<pre><code class="lang-angular2">EngineApp.java &#x6839;&#x636E;&#x53C2;&#x6570;&#x8F93;&#x5165;&#x9700;&#x8981;&#x8C03;&#x8BD5;&#x542F;&#x52A8;&#x7684;app
</code></pre>
<h2 id="&#x96C6;&#x7FA4;&#x6A21;&#x5F0F;">&#x96C6;&#x7FA4;&#x6A21;&#x5F0F;</h2>
<p>&#x793A;&#x4F8B;&#x5DF2;spark standalone&#x6A21;&#x5F0F;,&#x6BCF;&#x53F0;spark&#x8282;&#x70B9;&#x7EDF;&#x8BA1;&#x76EE;&#x5F55;&#x90FD;&#x9700;&#x8981;&#x6709;&#x76F8;&#x540C;&#x7ED3;&#x6784;&#x4E14;&#x76F8;&#x540C;&#x6587;&#x4EF6;&#x3002;</p>
<p>&#x96C6;&#x7FA4;&#x63D0;&#x4EA4;&#x7684;&#x683C;&#x5F0F;</p>
<ul>
<li>cluster<ul>
<li>config</li>
<li>lib &#x7A0B;&#x5E8F;&#x5305;</li>
<li>sh  spark&#x4EFB;&#x52A1;&#x542F;&#x52A8;&#x811A;&#x672C; </li>
</ul>
</li>
</ul>
<p>&#x4F8B;&#x5982;: A,B,C&#x4E09;&#x53F0;spark&#x96C6;&#x7FA4;,&#x6587;&#x4EF6;&#x653E;&#x7F6E;&#x76EE;&#x5F55; /home/easy-spark/cluster </p>
<h3 id="&#x96C6;&#x7FA4;&#x6253;&#x5305;&#x8981;&#x6C42;">&#x96C6;&#x7FA4;&#x6253;&#x5305;&#x8981;&#x6C42;</h3>
<pre><code>mvn clean install -Dmaven.test.skip=true -P cluster-build
</code></pre><h3 id="&#x63D0;&#x4EA4;&#x811A;&#x672C;&#x547D;&#x4EE4;">&#x63D0;&#x4EA4;&#x811A;&#x672C;&#x547D;&#x4EE4;</h3>
<p>&#x6CE8;&#x610F;:geo-json-app.sh&#x811A;&#x672C;&#x5185;&#x90E8;commonPropertiesPath&#x5FC5;&#x987B;&#x914D;&#x7F6E;&#x6210;&#x7EDD;&#x5BF9;&#x8DEF;&#x5F84;&#x5426;&#x5219;&#x8BFB;&#x53D6;&#x4E0D;&#x5230;&#x81EA;&#x5B9A;&#x4E49;&#x914D;&#x7F6E;&#x6587;&#x4EF6;</p>
<pre><code>$ cd cluster/sh
$ sh geo-json-app.sh
</code></pre>
<script>console.log("plugin-popup....");document.onclick = function(e){ e.target.tagName === "IMG" && window.open(e.target.src,e.target.src)}</script><style>img{cursor:pointer}</style>
						
					</section>
					
    </div>
    <div class="search-results">
        <div class="has-results">
            
            <h1 class="search-results-title"><span class='search-results-count'></span> results matching "<span class='search-query'></span>"</h1>
            <ul class="search-results-list"></ul>
            
        </div>
        <div class="no-results">
            
            <h1 class="search-results-title">No results matching "<span class='search-query'></span>"</h1>
            
        </div>
    </div>
</div>

				</div>
			</div>
			
		</div>

		
		
		
		<a href="develop-guide/" class="navigation navigation-next navigation-unique" aria-label="Next page: 开发指导">
			<i class="fa fa-angle-right"></i>
		</a>
		
		
		
	</div>
	<script>
		var gitbook = gitbook || [];
		gitbook.push(function() {
			gitbook.page.hasChanged({"page":{"title":"Introduction","level":"1.1","depth":1,"next":{"title":"开发指导","level":"1.2","depth":1,"path":"develop-guide/README.md","ref":"develop-guide/README.md","articles":[{"title":"文件读取示例hello-word","level":"1.2.1","depth":2,"path":"develop-guide/geo-json-app.md","ref":"develop-guide/geo-json-app.md","articles":[]}]},"dir":"ltr"},"config":{"plugins":["-sharing","-fontsettings","back-to-top-button","copy-code-button","cuav-chapters","heading-anchors","theme-fexa","popup","splitter","multipart@0.3.0"],"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"pluginsConfig":{"splitter":{},"search":{},"cuav-chapters":{"summaryMaxSize":20,"useLimitExpanded":false},"popup":{},"multipart":{},"theme-fexa":{"search-placeholder":"输入关键字搜索","logo":"./logo.png","favicon":"./favicon.ico"},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"heading-anchors":{},"highlight":{},"back-to-top-button":{},"copy-code-button":{},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"theme":"default","author":"shengshi_feiyang@yeah.net","pdf":{"pageNumbers":true,"fontSize":18,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":30,"left":30,"top":30,"bottom":50}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"variables":{"themeFexa":{"nav":[{"url":"https://gitee.com/shengshifeiyang/easy-spark","target":"_blank","name":"代码仓库"}]}},"title":"基于spark的快速离线分析框架使用文档","gitbook":"*"},"file":{"path":"README.md","mtime":"2020-11-09T05:41:05.015Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2020-11-11T07:40:22.004Z"},"basePath":".","book":{"language":""}});
		});
	</script>
</div>

        
    <script src="gitbook/gitbook.js"></script>
    <script src="gitbook/theme.js"></script>
    
        
        <script src="gitbook/gitbook-plugin-back-to-top-button/plugin.js"></script>
        
    
        
        <script src="gitbook/gitbook-plugin-copy-code-button/toggle.js"></script>
        
    
        
        <script src="gitbook/gitbook-plugin-cuav-chapters/cuav-chapters.js"></script>
        
    
        
        <script src="https://cdnjs.cloudflare.com/ajax/libs/anchor-js/1.2.1/anchor.min.js"></script>
        
    
        
        <script src="gitbook/gitbook-plugin-heading-anchors/anchor-style.js"></script>
        
    
        
        <script src="gitbook/gitbook-plugin-splitter/splitter.js"></script>
        
    
        
        <script src="gitbook/gitbook-plugin-search/search-engine.js"></script>
        
    
        
        <script src="gitbook/gitbook-plugin-search/search.js"></script>
        
    
        
        <script src="gitbook/gitbook-plugin-lunr/lunr.min.js"></script>
        
    
        
        <script src="gitbook/gitbook-plugin-lunr/search-lunr.js"></script>
        
    
        
        <script src="gitbook/gitbook-plugin-theme-fexa/fexa.js"></script>
        
    

    </body>
</html>

